{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from path_explain import set_up_environment\n",
    "from gradients import GradientExplainerTF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_up_environment(visible_devices='0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.random.randn(100, 2).astype(np.float32)\n",
    "y = x[:, 0] + np.random.randn(100) * 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fb070091250>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD4CAYAAADxeG0DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAUP0lEQVR4nO3db4xcV3nH8d+z60k6Doh1lW2rDDG2VOQ2iYlXbFOQX9XQOC2QbFwg0IKEQPKbIjVRtMgBRByJypYs/lQCiVoCoYooGEKyDbiVk8hIUSMSZZ11/pjEKEI48SQSRskCxZNmvH76Ync2s7P3ztyZuXfuPTPfjxQpntm9c0aEn46f85xzzN0FAAjXWN4DAAD0hyAHgMAR5AAQOIIcAAJHkANA4Dbk8aGXX365b9myJY+PBoBgnThx4jfuPtn6ei5BvmXLFs3Pz+fx0QAQLDM7E/U6pRUACBxBDgCBI8gBIHAEOQAEjiAHgMAR5AAQuFzaDwFglMwtVHXo2Gm9vFjTFRNlze7eppmpSmrPJ8gBIENzC1Xdcd8zqtWXJEnVxZruuO8ZSUotzCmtAECGDh07vRriDbX6kg4dO53aZxDkAJChlxdrXb3eC4IcADJ0xUS5q9d7QZADQIZmd29TuTS+5rVyaVyzu7el9hksdgJAkyw6TC7dMLZaJ9+0saQ7P3Q1XSsAkIW0O0xanydJr9cvpjPYJpRWAGBF2h0mg+hYkQhyAFgV10lS7bHDZBAdKxJBDgCr4jpJTMtlkrSel2bHikSQA8Cq2d3bZBGvu9RTOWQQHSsSQQ4Aq2amKvKY93oph8xMVXRgz3ZVJsoySZWJsg7s2Z5qx4pE1woArFGZKEfWxHsth8xMVVIP7lbMyAGgyaDKIWliRg4ATRqz5yyPnU2bucdVhBI+wOxKSf8h6U+1vCZw2N3/rd3vTE9P+/z8fF+fCwCDkPVZ4t0wsxPuPt36ehoz8guSbnf3J83srZJOmNlD7v7zFJ4NALkZxFniaei7Ru7ur7j7kyv//ntJz0kqzjcEgCZzC1XtPHhcW/cd1c6Dx9v2hw9qZ2a/Uq2Rm9kWSVOSHo94b6+kvZK0efPmND8WABLpdoY9qJ2Z/Uqta8XM3iLpR5Judffftb7v7ofdfdrdpycnJ9P6WABIrNsZ9qB2ZvYrlSA3s5KWQ/xud78vjWcCQNrizkyJm2GH0orYd2nFzEzStyU95+5f7X9IAJC+uYWqTIrcuRk3ww6lFTGNGvlOSZ+U9IyZnVx57fPu/l8pPBsAUnHo2OnIEDep7Qx7EDsz+9V3kLv7/0iR58wAQGHElU9cxWol7AVb9AGMhLjySaVgC5e9YIs+gCC17rj8m7+Y1E+fPxdby57dvW3dtWtFXLjsBUEOIDhR/eDfe+zF1fej+sNDWbjsBUEOIDhR/eCtGv3hzUEdwsJlL6iRAwhO0p2VRduBmRWCHEBwku6sLNoOzKwQ5ACCE7XjstWwLGQmQZADCE7UXZifeM/mzO/GLCoWOwEEaVgXLntBkAMovCLd0lNEBDmAQgvllp48USMHUGih3NKTJ4IcQKGFcktPnghyAIUWyi09eSLIARRaKLf05InFTgCFNsyHXaWFIAdQePSMt0eQAxg4+sLTRZADGCj6wtPHYieAgaIvPH0EOYCBoi88fZRWACTWqbYddY/mT556RYu1uiRp08aSJjaW9Nr5+rpn0xfeO4IcQCKdatud7tGUpNfO1zVmUmncVF/y1dfpC+8PpRUAiXSqbSe5R1OSLrp02SUbRvbs8CwwIweQSFwNu7pY09Z9R+WR70b7ba2uk3den87AwIwcQDLtatjdhHinZ6F7zMgBrBO1qDm7e9uaGnivSuNGPTxlzMgBrNFYtKwu1uRau6jZuCezV5s2lnTow9dSD08ZM3IAa7Rb1Hx03y7NTFW08+BxVbvo+65MlPXovl1pDxUrmJEDWCPJhp1uN++w2SdbqQS5mX3HzH5tZs+m8TwA2ZlbqGrnwePauu+odh48rrmF6pr3k1zkEPczZtGfyeJmttKakX9X0g0pPQtARuLq381hnuQih7if+ae/3swlEDlIJcjd/RFJr6bxLADZSXJg1cxUZXVRM27DTtzPfHlme8ffRfrMvdsO0JgHmW2R9BN3vybm/b2S9krS5s2b333mzJlUPhdAcu027lQmypwPXnBmdsLdp1tfH9hip7sfdvdpd5+enJwc1McCaBJb25bWlFtuPXJSO+56cF39HMVE1wowQqJq26bonZmLtfq6+jmKiSAHRkRjt2atvqTxlfaSykS57fZ6LnwIQ1rth/dI+pmkbWZ21sw+k8ZzAaSjuVtFkpbcV7tJOu3UpAe8+FLZ2enuH0/jOQC6k/QS43bdKp3OUKEHvPjYog8EpDm4JzaW9L+vX1D94nJxJOqih8bPxpVPXl6srQb/XT8+te7mHnrAw0CQA4FovYEn6rq05pp2kpMKG7PtmanKuvCnDTEcBDkQiKQ38Ly8WEv0s1Gz7UagIywEORCIpIuOY2ZtTyY0idn2kCHIgUBcMVFOdHTsUpvd2hwnO5zoIwcCEbWZpxssXA4vghwIQPNmnrGYo2KjcHjVaKC0AhRca7fKxS7OuaOMMhqYkQMFl7RbpdVEuZTBaFBEBDlQcN3cjdlQGjPtv/HqDEaDIqK0AhRQ88acbk2US9p/49XUw0cIQQ4UTGtNvFv/d+FiyiNC0RHkQEE0ZuHdlFKizhJvbNNnRj46CHIgZ3MLVX3+vqd1vt7dTHrTxlLkeSsSR8+OGhY7gRzNLVQ1e+9TiUJ83Gy1J/zrt+zQwpeujz1LnKNnRwszciBHh46dVn2pc2N4uTQeuaEn6ixxdnCOHoIcyFGSEkilzQFXjdc4ena0EeRAH6LO75aSB2ung7BKY9YxmDl6FtTIgR4134PpWt64M/vDpzR771NrXmt3E/3s7m0qjccfnlK/6Fx+jI6YkQM9ito6X484CKX51p79D5zSYu3NTpNNG0u65a+u1NGnX4ntQKku1rR131HKJojFjBzoUTctftXFmm49cnJNiEvL17UdeeIl3fmhq9veZp9kdo/RRZADPUqrxa++5Ku32Xc6b7x5dg80EORAj6KCtzRmbWvecRq32R/Ys331DPF2Pws0o0YO9Ciu9U+Sbv/BU22vXGvVepu9JO08eDyyo4XNPmhFkAN9iGv9u+3IycTPKI1b5AYeNvsgKYIcSEFrP/nbyqV1C5tRNm0s6c4PRR85y2YfJEWQA31qPXa2uljrWCf/+i07EgUym32QBEEOdOGLc8/onsdf0pK7xky6dMOYahEHXtWXXGZSVJl8olwinJEqulaAhL4494y+99iLq4uYF12RId7gvtzF0qxcGucKNqQulSA3sxvM7LSZvWBm+9J4JlA09zz+Uve/ZMsz8Mbxs1EnGAL96ru0Ymbjkr4p6W8lnZX0hJk94O4/7/fZQJF0007YUF9yXXbpBp288/oMRgQsS2NGfp2kF9z9l+7+hqTvS7ophecChdHPtng28CBraQR5RVLz3znPrry2hpntNbN5M5s/d+5cCh8LDE4/2+LZwIOsDWyx090Pu/u0u09PTk4O6mOBVPQ6q2YDDwYhjSCvSrqy6c9vX3kNGApzC1WNWbLzUybKpdWzUljcxKCk0Uf+hKR3mtlWLQf4xyT9YwrPBXLT2KlZXazJtHyMbCeN1kKCG4PWd5C7+wUz+6ykY5LGJX3H3U/1PTIgQ80bexoqTYdeNe/UTBLi7bbaA1kz76Glql/T09M+Pz8/8M/FaIi6R7M5YBsbe+KM2fJmn6RK46ZDH76WEEfmzOyEu0+3vs7OTgyVqHs0W2/V6bSxp5sQl968GALIC2etYKhE3aNZqy9p/wOnVmfpWfwdlF5x5Ikgx1CJC9TFWj3RsbJxGgue42aROzzpFUeeKK1gqGQRqJWJsr52yw796uAH9JWPXrvuejd6xZE3ZuQIXvPi5tvKJZXGTfWldAooJunRfbtW/8xlDygighxBa73UobV8YpLKpTGdjzhutjJRXg3pHXc9GFl6iZrhc9kDiobSCoIWtbjZzCW9fuFi5LngzeWQ/TdeTckEwWJGjqAl6Ra56Muz8j/ZeMm62+53Hjy++to/vLuinz5/jpIJgkOQI2hXTJRVTRDmf3hjSf9685vBHHXP5o9OVDkbBUGitIJCmluoaufB49q676h2Hjy+7jzwxvtJQrxh/wNvnhwR12/Oxh6EiCBH4XTandn8fjcWa/XVZ8SVZNjYgxAR5CicTrPlTgucnZ4txfebs7EHISLIUTidZsv9zJobvzu7extdKhgaBDkKJ25WPLGx1Pb9bp49M1XRgT3buQQCQ4FjbFE4cwtVzd77VOTuzE+8Z7Om3/HHazpOmpXGTZddskGLtfq6CyHKpXHCGkHjGFsEY2aqossuie6MbZwjfmDPdo1HXL9WX3JddukG/ergB/S1W3Yw48ZIoI8chfTbNicV3vXjU1r40vW67cjJyPcbdXC20mNUMCNHIbWrg792fjnk31Yudf27wDAiyFFInbpH5haq+sMbF9a9XhozOk8wcghyFNLMVEXlUvx/nrf/IHox9C1/tIFyCkYOQY7cxW3HP7DnXetOLWyIuqVHkhbP934LEBAqFjuRq6jDq247clLzZ17Vl2e2S1rejZl0Oz71cYwiZuTIVdR2e5d092Mvam6hqpmpih7dt0vR8/K12JmJUUWQI1dxM22X1pxEGDfTHjejTxwjj9IKUtd8h2a7CxrmFqrrdl82az5TZXb3tnW7OdmpCSwjyJGqqJr3Hfc9I0nrAvfQsdOxIS6tnYVz6TEQjyBHqtodQdsauu1OMYyqd7NTE4hGjRyp6HRjT1Rot6t7UzIBkiPI0bckN/ZEhXbcmeBf+ei1hDjQhb5KK2b2EUn7Jf2lpOvcnbNph1S7BcxON/bEtQVS9wbS0W+N/FlJeyT9ewpjQUF1WsDsdGPPpRvi/+JH3RvoX1+lFXd/zt25dnzIdbpDs9NuysVafc3lyQDSNbAauZntNbN5M5s/d+7coD4WKeh0h2ZUrbtVc/ADSFfHIDezh83s2Yh/burmg9z9sLtPu/v05ORk7yPGwMWd++2Sdh48Lklr7r+M08+lyQDidayRu/v7BzEQFFfEjWqrGvXyA3u269F9uyQptg2RA62AbNB+iI46HQ3bWjaJayvkQCsgG30FuZndbGZnJb1X0lEzO5bOsFAkSWbSzWWTmanKmlILB1oB2eqr/dDd75d0f0pjQUFFHVjVqjXsaSsEBoezVtBR88ad6mJt3YmFlE2AfBHkIyzpcbPS2hl2N78HIHsE+Yjq5rjZVpRNgGKha2VEddqtCSAcBPmIitucU12srbvNHkCxEeQjql1LoevNUgthDhQfQT6iOB8FGB4sdo6o1rPAk1yADKCYCPIR1tx9wvkoQLgorQyxxj2aSRYvOR8FCBcz8iHVbZ84164B4SLIh1S7PvEkuzcBhIPSypDqdKsPgOFBkA+puEVKFi+B4UOQDykWL4HRQY18SLF4CYwOgnyIsXgJjAaCPCCcAw4gCkEeiH7ODwcw3FjsDATnhwOIQ5AHgr5wAHEI8kDQFw4gDkEeCPrCAcRhsTMQ9IUDiEOQB4S+cABRKK0AQOAIcgAIHEEOAIEjyAEgcH0FuZkdMrPnzexpM7vfzCbSGhgAIJl+Z+QPSbrG3d8l6ReS7uh/SACAbvTVfujuDzb98TFJH+5vOMOLkwsBZCXNPvJPSzqS4vOGBicXAshSx9KKmT1sZs9G/HNT0898QdIFSXe3ec5eM5s3s/lz586lM/pAcHIhgCx1nJG7+/vbvW9mn5L0QUnvc3dv85zDkg5L0vT0dOzPDSNOLgSQpX67Vm6Q9DlJN7r7+XSGNHw4uRBAlvrtWvmGpLdKesjMTprZt1IY09Dh5EIAWeq3a+XP0xpIUaXRbcLJhQCyxOmHbaTZbcLJhQCywhb9Nug2ARACgrwNuk0AhIAgb4NuEwAhIMjboNsEQAhY7GyDbhMAISDIO6DbBEDRUVoBgMAR5AAQOIIcAAJHkANA4AhyAAgcQQ4AgSPIASBwBDkABI4gB4DAEeQAEDiCHAACF9RZK2lcuwYAwyaYIE/z2jUAGCbBlFa4dg0AogUT5Fy7BgDRgglyrl0DgGjBBDnXrgFAtGAWO7l2DQCiBRPkEteuAUCUYEorAIBoBDkABI4gB4DAEeQAEDiCHAACR5ADQODM3Qf/oWbnJJ0Z+AcP3uWSfpP3IAaI7zu8Rum7SsX9vu9w98nWF3MJ8lFhZvPuPp33OAaF7zu8Rum7SuF9X0orABA4ghwAAkeQZ+tw3gMYML7v8Bql7yoF9n2pkQNA4JiRA0DgCHIACBxBnjEzO2Rmz5vZ02Z2v5lN5D2mLJnZR8zslJldNLNg2re6YWY3mNlpM3vBzPblPZ4smdl3zOzXZvZs3mPJmpldaWY/NbOfr/w3/C95jykpgjx7D0m6xt3fJekXku7IeTxZe1bSHkmP5D2QLJjZuKRvSvo7SVdJ+riZXZXvqDL1XUk35D2IAbkg6XZ3v0rSeyT9cyj/2xLkGXP3B939wsofH5P09jzHkzV3f87dT+c9jgxdJ+kFd/+lu78h6fuSbsp5TJlx90ckvZr3OAbB3V9x9ydX/v33kp6TFMRNNgT5YH1a0n/nPQj0pSLppaY/n1Ug/2dHcma2RdKUpMfzHUkyQV31VlRm9rCkP4t46wvu/p8rP/MFLf/V7e5Bji0LSb4vECoze4ukH0m61d1/l/d4kiDIU+Du72/3vpl9StIHJb3Ph6Bxv9P3HXJVSVc2/fntK69hCJhZScshfre735f3eJKitJIxM7tB0uck3eju5/MeD/r2hKR3mtlWM7tE0sckPZDzmJACMzNJ35b0nLt/Ne/xdIMgz943JL1V0kNmdtLMvpX3gLJkZjeb2VlJ75V01MyO5T2mNK0sXH9W0jEtL4b9wN1P5Tuq7JjZPZJ+JmmbmZ01s8/kPaYM7ZT0SUm7Vv6/etLM/j7vQSXBFn0ACBwzcgAIHEEOAIEjyAEgcAQ5AASOIAeAwBHkABA4ghwAAvf/j3ETY9eWjMAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(x[:, 0], y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.Sequential()\n",
    "model.add(tf.keras.layers.Input(shape=(2,)))\n",
    "model.add(tf.keras.layers.Dense(8))\n",
    "model.add(tf.keras.layers.Activation(tf.keras.activations.relu))\n",
    "model.add(tf.keras.layers.Dense(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'dense/kernel:0' shape=(2, 8) dtype=float32, numpy=\n",
       "array([[-0.06247556,  0.15779036,  0.6914294 ,  0.26529598, -0.6993861 ,\n",
       "         0.25997686, -0.6677437 ,  0.04861528],\n",
       "       [ 0.0569433 ,  0.67670095,  0.27014756, -0.72139126,  0.00326014,\n",
       "        -0.5402215 , -0.13996637, -0.25491858]], dtype=float32)>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers[0].weights[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 8)                 24        \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 8)                 0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 9         \n",
      "=================================================================\n",
      "Total params: 33\n",
      "Trainable params: 33\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 100 samples\n",
      "Epoch 1/10\n",
      "100/100 [==============================] - 0s 3ms/sample - loss: 0.3182\n",
      "Epoch 2/10\n",
      "100/100 [==============================] - 0s 56us/sample - loss: 0.1725\n",
      "Epoch 3/10\n",
      "100/100 [==============================] - 0s 56us/sample - loss: 0.1052\n",
      "Epoch 4/10\n",
      "100/100 [==============================] - 0s 53us/sample - loss: 0.0674\n",
      "Epoch 5/10\n",
      "100/100 [==============================] - 0s 35us/sample - loss: 0.0517\n",
      "Epoch 6/10\n",
      "100/100 [==============================] - 0s 35us/sample - loss: 0.0426\n",
      "Epoch 7/10\n",
      "100/100 [==============================] - 0s 38us/sample - loss: 0.0377\n",
      "Epoch 8/10\n",
      "100/100 [==============================] - 0s 38us/sample - loss: 0.0348\n",
      "Epoch 9/10\n",
      "100/100 [==============================] - 0s 38us/sample - loss: 0.0325\n",
      "Epoch 10/10\n",
      "100/100 [==============================] - 0s 38us/sample - loss: 0.0307\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fb0287b7110>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=0.1),\n",
    "              loss=tf.keras.losses.MeanSquaredError())\n",
    "model.fit(x, y, batch_size=50, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model.predict(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fb0280ff7d0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD4CAYAAADxeG0DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAWkUlEQVR4nO3dfYxcZ3XH8d/ZyQTGAWWMsmrlwcYRpY4SDNmyBSP/0wRUh7dkMS8hBSREVf9TpCaKjBwFKVGb1pYsXiqBRC2BEIoVHOFkiRSqJciRUFEdscYLibGNItq8TKgwijeAPU725fSP3RnPzt47c2fm3rn3znw/kqV4dvbOXUF+++Tc85zH3F0AgPwaS/sGAAD9IcgBIOcIcgDIOYIcAHKOIAeAnLsijQ+95pprfOvWrWl8NADk1okTJ37v7uOtr6cS5Fu3btXs7GwaHw0AuWVmzwW9TmkFAHKOIAeAnCPIASDnCHIAyDmCHAByjiAHgJxLpf0QAIbB9MmqDs6c1UvzNW0ql7R31zZNTVQGfh8EOQD0YPpkVfc88rRqC0uSpOp8Tfc88rQkDTzMKa0AQA8OzpxthHhdbWFJB2fODvxeCHIA6EF1vtbV60mitAIAPSiYaSnghLWC2brXkq6lE+QA0IOgEA96fRC1dEorANCDSrkU6fVB1NIJcgBDbfpkVTsPHNO1+x7XzgPHNH2yGst19+7aplKxsOa1UrGgvbu2rXntpZCaedjrvaC0AmBoJVnWqH9/p9r3pnIp8AHoppAVfS8IcgBDq11ZI4769NREpeN19u7atuaXiRS8cu8HQQ5gaA2irNFJ1JV7PwhyAENrEGWNKKKs3PvBw04AQyvqA8m8Y0UOYGgNoqyRBX0HuZltlvRdSX8mySUdcvd/7/e6ABCHpMsaWRDHinxR0t3u/nMze6OkE2b2hLv/KoZrAwA66DvI3f23kn67+s9/NLPTkiqSCHIAQyUr88dbxVojN7OtkiYkPRXwtT2S9kjSli1b4vxYACNuEAGbpfnjrWLrWjGzN0g6KulOd/9D69fd/ZC7T7r75Pj4eFwfC2DE1QO2Ol+T63LARtmK3832/SzNH28VS5CbWVErIX7Y3R+J45oAEEWvAdvtL4AsbC4K03eQm5lJ+pak0+7+lf5vCQCi6zVgu/0FELaJaNCbi4LEsSLfKemzkm42s7nVPx+M4boA0FGvAdvtL4Asby7qO8jd/b/c3dz9He5+4+qfH8ZxcwDQSa8B2+0vgKmJivbv3q5KuSTTytzx/bu3p/6gU2JnJ4Ah8Lorxhplko0birrvIzckMpUwq5uLCHIAudXaEihJlxaWI31fvUZeP3uzkqG+8G4xNAtAbvXSsdLcrSKtnLFZX4nnMcQlghxAjvXSsZLlfvBeEeQAcquXjpUs94P3iiAHkFu9dKxkuR+8VwQ5gNzqpSUwy/3gvaJrBUCuddsSOIyHTRDkAEZOVvvBe0VpBQByjiAHgJwjyAEg56iRA8icrB6pllUEOYBMyfKRallFaQVApgzjFvqkEeQAMmUYt9AnjSAHkCnDuIU+aQQ5gEwZxi30SeNhJ4BMGcYt9EkjyAEkqpdWwmHbQp80ghxAYmglHAyCHEDXoq6y27USEuTxIcgBRFIP7+p8TSbJV19vt8qmlXAw6FoB0FHrgcXe8vWwDTu0Eg4GQQ6go6ASSaugVTathINBaQVAR1FKIeUNRe08cCywbk4rYbIIcgAdlTcUdf7iQujXx0w6f3Gh8Z7WujnBnSxKKwDamj5Z1Z8uLYZ+/aorC1puLZqLQVeDRJADaOvgzFktBCR1uVTU/x74kC4tLId+b5XulIEgyAG0FVYff6W2UkZZ8oDl+KqCWSL3hLWokQMjrtPmnk3lUuDKut5CWDALDfN2IY/4sCIHRlhzf7jr8kPK6ZPVxns6tRDe8Z7Nodev0C8+ELEEuZl928x+Z2bPxHE9AIMRtoX+ziNz2nngmKZPVjU1UdH+3dtVKZdkWgnn/bu3N1btD0xt1863vmndtekXH5y4SivfkfR1Sd+N6XoABqBdf3g3LYSH/+G9HJicoliC3N1/YmZb47gWgMEJq3/XdTPgin7x9AysRm5me8xs1sxmz507N6iPBdBGUP27FQOusm9gQe7uh9x90t0nx8fHB/WxANporn+HYcBV9tF+CIyo1pr2Z3Zs0dET1TUPP3lgmQ8EOTCCgk7uOXqiqo+9q6Inz5wLfGDJw8zsiiXIzewhSX8j6Roze1HSfe7+rTiuDaB7nUI3rO3wyTPn9NN9NwdejyPbsiuurpU74rgOgP5FCd1uT+7hyLZsY2cnMGTahW5dtyf3cGRbtlEjBwag1/pyL98XJXT37tq2ZtUutX+w2WneCtJFkAMJ67W+HPX7WsM+7BCI5tDt9uSeboMfg0WQAwnrtb4c5fuCwr44ZioWTAtLlycPBoVuNzsxObIt2whyIGG91pejfF9Q2C8su8qloq563RXrQrefFkK24GcXQQ4krNf6ctj3jZk1phK2O/Rh7r6/XfMaLYTDi64VIGE3XRc8kqI6X2uMig0SNgdlyb0xM7yb7pMo3SzIJ1bkQMKePBM+JK7dqrj+97sf/sW6k3bqAdzuIWRrGSVsyiEthPnHihxIWKegbLcqnpqoaDnkuLSX5muhhz5IWnfyT9jpmbQQ5h8rciBhnWZ+S+3DvlONvbWj5ODMWV14dXFdGSXo1wEthMOBFTmQsCgzv9utijudmRl07uZ8bX0feV19Zd56ZBvyyzyFU64nJyd9dnZ24J8LpKVer66XOJr/rSsVC41ySHNN+6brxhuTCMsbinJf6UZp/dpYm1Psw1TKpcDhWMg2Mzvh7pOtr1NaAQaguQc7qJdb0rrWwAePP9/4/vMXF1QqFvTV229c995uQ1ziAeewIciBAQvaWLPzwLF1Ne1WtYUl3f/YKf3x0mKk8N64oag/1ILfywPO4UKQAxkQdYXcrvbdrFQs6L6P3CBJzEgZAQQ5kAFROls6KZhp2T30IAlmpAwvghzIgKCNPd0wSV/+5DsDA5oZKcOPIAdS0vrQs/W8zObOlE3lki6+thg4nlZa6YIhrEcXQQ7EKOp0wbDDj9v1dU+frOquI3OBG3sqPLwcaWwIAmIStDGnPtyqVdgAqzuPzIUO0pqaqOjTO7as22rPw0sQ5EBMupku2K5LpTpf051H5jTxzz9aF+gPTG3XV2+/cd1sFcoqo43SCtBGNwcxdHOARJQulfMXFwInI/LwEq1YkQMhuimVSOGbbK4uFbXzwDFdu+/xRtkkyvwViXnhiIYgB0J0exBDUDgXx0wXXltc88vgriNzmn3u5cb42U7YTo9OCHIgRLdnbTbPBpcks5XzM5sPQZZWWgUPr85R+em+m/W1229suzpnOz06IciBEN0co1Y3NVHR3l3bVCyY2o1Dcamxsq//AiiXiuveR0cKouBhJxCi3TFqdUEPQw/OnF23Cg/SvLKvP8Ds55R7jC7mkWNk9BKSrd/TvNvy6lJRF15bXBPapWIh8jZ7ZoKjW8wjx0gL2kkZduhxs9Y54s3XCJpEWFtYUiHCQQ+UTBAnghwjIawD5d5Hn26cUl8w0x3v2awHprZHvkaQJXcVCxZaXimXirr/1hsomSA2POzESAjrNLnw2lJj9bzkrgePP68vTT/d1TVaVcolHfz4O7Vxw9qHlxs3FPW122/U/bfeoIMzZ9f0lQP9YEWOkdDNvO+HnnohcFUe5Rr1kknY7steSzxAO7GsyM3sFjM7a2bPmtm+OK4JxCnqTkop/AzMvbu2rRtY1axg1nHuSbebjIAo+g5yMytI+oakD0i6XtIdZnZ9v9cF4tS8Wac+bCoslAsW/JWpiUrgCNm6ZfeOq+puNxkBUcRRWnm3pGfd/TeSZGbfk3SbpF/FcG0gNq3lji9NP73mpPq6O96zOfQalTbllSg7MMPKM+zeRD/iKK1UJL3Q9PcXV19bw8z2mNmsmc2eO3cuho8F+vPA1HZ9ZseWxgq8YKbP7Niyrj4+fbLaGHp14dXF0OvddN14x88MKvHQioh+Dexhp7sfknRIWtkQNKjPBVq1bvIJO+uy/t5OveN1T57pvECpfw67NxGnOIK8Kqn5v0XfvPoakDlhXSOzz7285nzMerhG7R2Xote5mSeOuMUR5D+T9DYzu1YrAf4pSX8Xw3WB2IV1jRw+/nzjQWZ1vqa93/+F7n/sVNsVeCvq3EhL30Hu7otm9gVJM5IKkr7t7qf6vjMgAWGr5tZa38KStw1xa/ke6txIUyx95O7+Q3f/S3d/q7v/axzXBJIQx6q5VCzo0zu2cG4mMoMt+hgpQV0j7Tb5tCqXinp9cUwPHn9e//fKpbZ95cCgEOQYKUEbgz69Y0ukXZ/lUlGvLi7r/MWVkkt9B2inszyBpDFrBSMnqGtk8i1varQEljcU9adLi1pYXjtn3EyhHSz1bfaUV5AGghyZFOUQiDhP02kN96Br33Vkru012GaPtBDkyJwoEwLjmiIY9ssgaNV+cOZs2+mHtB8iLdTIkTlRJgSGvef+x041ttN3mvVd/2VQna/J1bnW3W6CIu2HSBNBjsyJMiEw7D3ztYXIwdztSNnmB6XS5SmJtB8ibZRWkDlRJgRGPSii3UPIXkbKsr0eWcSKHJkTZUJgNwdFhAVzueUotjpq3cgbVuTInCgTApvfU52vrdsy3ywomKdPVvXKxfVb8IsFo9aN3DEPOdYqSZOTkz47Ozvwz8Vw2nngWGiZpVQsaP/ulfnizb8YXr7wqmoLywHvH9Ppf/lAovcL9MrMTrj7ZOvrrMiRe+1q2vUQb21VDBMU7kDWUSNH7oXVtCvlUtczxYE8IsiRe50ejnaz43JjyANQIMsorSC3mndlXr06lXD+4sK6h6NRWxWLBdN9H7kh6dsGYkeQI5fCztLcuKGom64b18GZs7rryJw2lUu66bpxHT1RXVNeKRZMxTHTxdWaeLlU1P233kCPOHKJIEeqWlfVZgpcVbcKq3ufv7igB48/3/h7db6moyeq+ti7Ko0zOevTDS82Pdh8dZGHnMgvghypaXdCffMQLGl9T3k3de/awpIeeuoFffmT79TUREU7DxxrzBRvfg9jaJFXBDlS06mbpLawpLsentMVY6aFpbWHOLy+ONZVq+CSe+MXQy9b84Eso2sFqYkSnO5qhHhdbWGpp37v+qo7rF2RrfnIK4IciZk+WW07UjaN4HxpvhZplguQJ5RWkIhOBz9Mn6zqwquLA7+vTaubhKT2s1yAPCHIkYhOs76bQ75uQ3FMtcVlxTH+pzhmkq0tyzSvuhlHi2FCkCM2za2EYVlcna/p7od/0TiBvtnGq16nf9u1LTDkW7VOOywWTFddeYVeqV1uXZRYdWM0EOSIRWspJYxJgSEurdSvW8seYSfa/9WWq3X8N+e15K6CmW7/6816YGr7umsS3BgFBDliEWUwVbuZ4dLlh5+dTrSv79Ss/0JYctfRE1VNvuVNBDdGEkGOWLRrJTR1nnfSrmukNdh3HjgWWn8nyDGKaD9ELNqNkv3q7Te2/d6CWVeHF7OhB1iLIEcswnqzb7puXPc88nTbE3zqW+ejYkMPsBZBjlhMTVS0f/d2VcolmVZW4vt3b9eTZ86F1s7r7+m2HMKGHmAtauSITVBv9l1H5gLfa5J+uu/mnj9HorUQqCPIkaiwh5z9lkHY0ANc1ldpxcw+YWanzGzZzNad7Izh1WmOSh1lECB5/a7In5G0W9J/xHAvyIlOc1SaUQYBktdXkLv7aUkys3juBrnQbo5KUEBTBgGSNbCuFTPbY2azZjZ77ty5QX0sEkAfN5AtHYPczH5sZs8E/Lmtmw9y90PuPunuk+Pj473fMVIX9qCyvKE44DsBIEUIcnd/v7u/PeDPDwZxg8ievbu2qVhYX07706XF0IeeAJLDhiB01NqhIklXXbn+8crCsjfmjQMYnH7bDz9qZi9Keq+kx81sJp7bQlbUO1SqqzPG6x0qzSfeN6NODgxev10rj0p6NKZ7QQpaR8S2tgaGdaiEYd4JMHjs7BxhUfrBu1lhs9EHSAc18hHW6VxNqfMKu2C2ZkgW/eLA4LEiH2FR+sH3djhDc9ld/3PgQ4ncH4BoCPIR01wTHzMLPD+zeRVeX2GHHZhMTRxIH0E+Qlpr4kHBHFTnrod568qcmjiQDQT5CAk7ILlgpmX3tgOtGH4FZBdBPmTatROG1cSj1rkZfgVkE10rQyRs80592zxnXQLDiSAfIp3aCTnkARhOlFaGSKd2QurcwHAiyIdIlPMxqXMDw4fSyhChdAKMJlbkQ4TSCTCaCPIhQ+kEGD2UVgAg51iRZ1inWeEAIBHkmRVlVjgASJRWMivKrHAAkAjyzIoyKxwAJII8s5iLAiAqgjyj2NwDICoedmYUm3sAREWQZxibewBEQWkFAHKOIAeAnCPIASDnCHIAyDkedsaEuSgA0kKQx4C5KADSRGklBsxFAZAmgjwGzEUBkCaCPAbMRQGQpr6C3MwOmtkZM/ulmT1qZuW4bixPmIsCIE39rsifkPR2d3+HpF9Luqf/W8qfqYmK9u/erkq5JJNUKZe0f/d2HnQCGIi+ulbc/UdNfz0u6eP93U72RG0rZC4KgLTE2X74eUlHwr5oZnsk7ZGkLVu2xPixyaGtEEAedCytmNmPzeyZgD+3Nb3nXkmLkg6HXcfdD7n7pLtPjo+Px3P3CaOtEEAedFyRu/v7233dzD4n6cOS3ufuHtN9ZQJthQDyoN+ulVskfVHSre5+MZ5byg7aCgHkQb9dK1+X9EZJT5jZnJl9M4Z7ygzaCgHkQb9dK38R141kEcetAcgDhmZ1QFshgKxjiz4A5BxBDgA5R5ADQM4R5ACQcwQ5AOQcQQ4AOUeQA0DOEeQAkHO52hAUdTY4AIyS3AQ5s8EBIFhuSivMBgeAYLkJcmaDA0Cw3AQ5s8EBIFhugpzZ4AAQLDcPO5kNDgDBchPkErPBASBIbkorAIBgBDkA5BxBDgA5R5ADQM4R5ACQcwQ5AOScufvgP9TsnKTnBv7Bg3WNpN+nfRMDws86nPhZs+ct7j7e+mIqQT4KzGzW3SfTvo9B4GcdTvys+UFpBQByjiAHgJwjyJNzKO0bGCB+1uHEz5oT1MgBIOdYkQNAzhHkAJBzBHmCzOygmZ0xs1+a2aNmVk77npJiZp8ws1NmtmxmuW3jasfMbjGzs2b2rJntS/t+kmJm3zaz35nZM2nfS9LMbLOZPWlmv1r9/+8/pX1PvSDIk/WEpLe7+zsk/VrSPSnfT5KekbRb0k/SvpEkmFlB0jckfUDS9ZLuMLPr072rxHxH0i1p38SALEq6292vl7RD0j/m8X9XgjxB7v4jd19c/etxSW9O836S5O6n3f1s2veRoHdLetbdf+Pur0n6nqTbUr6nRLj7TyS9nPZ9DIK7/9bdf776z3+UdFpS7k6vIcgH5/OS/jPtm0DPKpJeaPr7i8rhv/AIZ2ZbJU1IeirdO+lero56yyIz+7GkPw/40r3u/oPV99yrlf+EOzzIe4tblJ8VyCMze4Oko5LudPc/pH0/3SLI++Tu72/3dTP7nKQPS3qf57xpv9PPOuSqkjY3/f3Nq68h58ysqJUQP+zuj6R9P72gtJIgM7tF0hcl3eruF9O+H/TlZ5LeZmbXmtmVkj4l6bGU7wl9MjOT9C1Jp939K2nfT68I8mR9XdIbJT1hZnNm9s20bygpZvZRM3tR0nslPW5mM2nfU5xWH1p/QdKMVh6IPezup9K9q2SY2UOS/lvSNjN70cz+Pu17StBOSZ+VdPPqv6NzZvbBtG+qW2zRB4CcY0UOADlHkANAzhHkAJBzBDkA5BxBDgA5R5ADQM4R5ACQc/8PlAKHgnFBxVcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(y, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer = GradientExplainerTF(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "attributions = explainer.attributions(x,\n",
    "                                      multiply_by_input=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fb0207602d0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD4CAYAAADxeG0DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAATyElEQVR4nO3df4jcd53H8dc70+ndROU2R4PSbWMKJ7lLjTZ0kUr+OC4WU8XWWO1pTwVRyD8K115ZSa/lGg+hhYAoVPByZxG5nPanY4+epJUWypVG3DiJaa6NeIK1U8FIXZXL3jnZvO+P3dnuzs73O9/Z+Xy/3/l85/mAQndmd+YztHntJ+/P+/P5mLsLABCvTWUPAAAwGoIcACJHkANA5AhyAIgcQQ4AkbukjDe97LLLfPv27WW8NQBE68SJE7929629j5cS5Nu3b9fc3FwZbw0A0TKzn/d7nNIKAESOIAeAyBHkABA5ghwAIkeQA0DkCHIAiFwp7YcAMCmarbYOHzurV+cXdPlUQ7P7dmj/7umg70GQA0BOmq227nzstBY6i5Kk9vyC7nzstCQFDXNKKwCQk8PHzq6EeNdCZ1GHj50N+j4EOQDk5NX5haEe3yiCHABycvlUY6jHN4ogB4CczO7boUa9tuaxRr2m2X07gr4Pi50AkJPugiZdKwAQsf27p4MHdy9KKwAQOYIcACJHaQUAUhSxM3NUBDkAJChqZ+aoKK0AQIKidmaOiiAHgARF7cwcFaUVAOjRrYt7wvOhd2aOiiAHgFV66+K98tiZOSqCHABW6VcX75quateKmV0p6ZuS3izJJR1x96+M+roAUIak+rdJeu7g3mIHk1GIxc4Lku5w952SrpP0WTPbGeB1AaBwRZ1YGNLIQe7uv3T3Hy3/++8lvShpvP7eAQAZFXViYUhB2w/NbLuk3ZJ+0Oe5A2Y2Z2Zz586dC/m2ABDUH9dfj8apRl333rxr7OriqwULcjN7o6RHJd3m7r/rfd7dj7j7jLvPbN26NdTbAkAw3Y6V35zvrDz2fxculjiibIJ0rZhZXUshftTdHwvxmgBQtLSdnKPMyPM+ryVE14pJ+rqkF939S6MPCQDKkcdOziLOawlRWtkj6ZOS9prZyeV/3h/gdQGgUHl0rBRxXkuIrpX/dHdz93e4+zXL//xHiMEBQJHy6Fgp4rwWdnYCmBiDatV53LF5+VRD7T6hHbIvnSAHMBGy1qpD37E5u2/HurNbQvelc4wtgIlQ1tni+3dP696bd2l6qiHT0nktofvSmZEDmAhlni0eepbfixk5gIkQ4xkqWRHkACZCjGeoZEVpBUAllNGRMi4IcgDRK6sjZVxQWgEQvVhuu88LM3IAUVpdSkm6JHncbrvPC0EOIDqDLkjuqkJHShaUVgBEJ+2C5K6qdKRkwYwcQHTSSiYmVaojJQuCHEB0kg6imp5qDLzpPu9LHspAaQVAdDa6uadbW28vL5B22xSbrXaOo80fQQ4gOhs9iKqqbYqUVgBEaSObe8o8OCtPBDmAwpVVpy7ikocyUFoBUKgy69RVPTiLIAdQqDLr1EVc8lAGSisAClV2nbqKB2cxIwdQqCpf8FAWghxAoapapy4TpRUAhdrIBQ9V3I0ZEkEOIJisgTtMnTrrpRGTjNIKgCDyaius6m7MkAhyAEEkBe6hx8+M9Lpld7nEgCAHEERSsM4vdEaaldPlMhhBDiCItGAdpQxCl8tgBDmAINKCdZQySFV3Y4ZE1wqADevtUnnDpTX9zx/WX8E2ahmkirsxQyLIAWzI3c3TOnr85ZUb7NvzC6pvMtVrps7i6/faUwbJH6UVAENrttprQryrc9H1hksvoQxSMGbkADLrllL6nend9duFjk7e896V7z/0+Bnd9uBJSdKWzXXdc+PVBHtgQWbkZvaAmf3KzF4I8XoAxs/qDT9puvXwZqut2YdPaX6hs/Lcb853NPvIqejvyBw3oWbk35B0v6RvBno9ACXqt9W+34afXqbXu1cOHzurzsXe4ovUWXQdPnaWWXlAQYLc3Z81s+0hXgtAuZLONskS4h+/bttKQKe1HLIrM6zCauRmdkDSAUnatm1bUW8LYEhJW+1rZlr09TNsaWlRs/eArKT7MbvPIZzCulbc/Yi7z7j7zNatW4t6WwBDSpotL7r33WH55Y9eo+cO7l1XKpndt0P1Tbbudeo1ox0xMNoPAayRNFvuthJmbS3cv3tah295p6Ya9ZXHtmyu6/BH3kl9PDDaDwGsWdz8k0Y9cVPPsDss2ZFZjFDth9+S9LykHWb2ipl9JsTrAshf7zni8wsdyZdmz2zqiUOorpVbQ7wOgPwk3d7Tb3Gzc9G1+dJL1PqH95Y0WgyD0gowAdKuS+Pihvix2AlMgLTr0ri4IX4EOTAB0mbdXNwQP4IcmABps24ubogfNXKgorK2FEq0CcaOIAcqqHdxc36ho/om05bNdc2f76zpWun9uX6dLRhvBDlQQRtpKUzrbCHMxxtBDlRA70w66bCqtJbCtM4Wgny8EeRA5JqttmYfObVS/067+CGtpZB+8ngR5EDEmq22bn/opBJOl12ju7iZVAdPmsnTTz7+aD8EItWtaaeFeG9LoaQ156p06+DNVpt+8ogxIwcisno2vSnlooeu5w7uXfP1nvueTqyDd7+XrpX4EORAJHq7SgaF+OpzwLsG1cHpJ48TpRUgElkuP+6qbzIduunqdY9zrko1EeRAJLJ2j2zZXNfhW/rfwkMdvJoIciASSbNmW3Ut5pbNdd1z49Wp169xrkr1mGfpWwpsZmbG5+bmCn9fIBb9WgQlramRS0sXGcuXdm12Neo1wrmizOyEu8/0Ps6MHBgjzVZb13zhSd324Ml1LYKS9OFrp1VbNQXvLPqaEJde70LB5KBrBRgTvV0pqy10FjNv/JHYjTlpCHIgR1lPE2y22rrjoVOpLYXDVEHpQpksBDmQk6ynCd7dPK2jx19WqNUqulAmDzVyICdppwl2NVvtoCFOF8pkYkYO5CSpTt2eX9Ce+55e2WYfIsTpVJlsBDkQWLcunhbQ3VMGB22zz2KaM1EmHkEOBJTWeRJSvWY6/JH+uzcxeaiRAwENcx5KFtbnsalGnRDHGszIgYDSbudJY7a+vbBRr+nD107rmZfOcawsUhHkQADNVluHHj+zoZ/tlkkkzgLHxhDkwIiarbZmHz61bqt8Vm+49JKVwCa4sRHUyIERHXr8zIZDXJJ+u9AJOBpMIoIcGNH8iEHMdnqMiiAHNqh7UmGaes3UqCf/MWM7PUKgRg5sQLPV1t89eFIXU75nk2ldm2DWQ7SAYRDkwJCarbZuf/Bk6s7NpA07XG6MPAQJcjO7QdJXJNUk/Yu73xfidYFx8/F/fl7P/fdrA7+PDTso0sg1cjOrSfqqpPdJ2inpVjPbOerrAuPm7ubpTCE+PdUgxFGoEIud75L0U3f/mbv/QdK3JX0wwOsCY+XffvBypu9j8RJFCxHk05J+serrV5YfW8PMDpjZnJnNnTt3LsDbAsVottra/Y9PKkur+Ceu28ZsHIUrbLHT3Y9IOiJJMzMzoc7RB3I1zGmGX/7oNYQ4ShEiyNuSrlz19RXLjwHR6rYJZj0Ei5k4yhQiyH8o6W1mdpWWAvxjkv4mwOsCpRj2TPFPXLdNX9y/K+dRAclGDnJ3v2Bmn5N0TEvthw+4+8aOgQMK1m+DTtYzxbleDePCPMBVU8OamZnxubm5wt8XWK3f7fWNei1TiE816jp009WEOAplZifcfab3cXZ2YiLd3Tytfz2+vp1wobOomlniXZrcj4lxRJBj4jRbbR3tE+Jdi+4yad1MnTIKxhVBjokz6IZ7aSnEu2HOLBzjjiDHRFi9qJl1Vagb4s8d3Jvn0ICREeSonGarrS/8+xn95vzShQ+N+iZduOjqLA6/sP/qBi9TBopEkKNSmq22Zh85tSa0Fzppp4an4/YexIAgRyUMuxMzKw7AQgwIckSt2Wrr7x/7sc6PMOtOsmVznQVORIE7OxGtZqut2YdPbSjE6zXTVKMu09LmnnrN1jzfqNd0z41XBxopkC9m5IjWocfPqJPlbNke/doJuUsTMSPIEaW7m6c1v9AZ+ueS2gm5SxMxo7SC6AzamZmGdkJUEUGO6GTZmZmEdkJUEaUVjKW0mnXarHqTSV/662skad2Z4o16jXZCVBJBjrHTe7FDe35Btz94UnM/f01f3L9Ll081+vaLm5ZCfHWtmwVMTAKCHGOn38UOLuno8Zc189Y/1ey+Hetm2ybp4z3XrbGAiUlBkGPsJJVOXEsh3+06YbYNLCHIMRZW18Q3pVzs0J5f0FUHnyC8gVUIcpSutyaeFOJdrqVAv/Ox05LExh5MPNoPUbqslx33Wugs6vCxsytfd38htJfPHO+GfbPVDjhaYPwQ5ChdWjvhls31zD/b7xdCb9gDVUSQo3RJm3SmGnX974ADsVb/bNIvBHZzouoIcuSi2Wprz31P66qDT2jPfU+nljdm9+1Qo15b81ijXpOZUksuvRt8kn4hsJsTVUeQI7hhatXdxcmFzqJqtnSU7PRUQ/fevEvz55MPxep+z+qFzKRfCOzmRNUR5Agua616deBLS90q3eDdv3s6cSbdPcGwtxtl/+5p3XvzLk1PNWTqH/ZAFdF+iOCSrlvrrVWnBf7+3dN9d3AOmmGzmxOTiCBHUM1WWyb1PZ2wd4adtAjZ/UXQDWT6woF0BDmCGHT5sWn9RcZph181W+2V2TXBDaQjyDG03t2Tf/XnW/XoiXZqh4lL6wJ5dt8O3f7gyXWz9+6ZKgQ4kA2LnRhKv46Uo8dfHrgzc7rPwuX+3dOJF0TQ+w1kx4wcmaSVTgbd1pO2QDmdUF7p1tM5OwUYjBk5BuptExzGoBbAtN5vzk4BsmFGjhVJs98sh1r1dqo06rVMPdxpnSl77ns6tT0RwJKRgtzMbpF0SNJfSHqXu8+FGBSK1+96te4xsYNm4o16TR++dlrPvHRuQyWQpM4Uzk4Bshl1Rv6CpJsl/VOAsaBEaZtzaikXPUznWLdOak/k7BRgrZFq5O7+ortzRmgFpM1+0y566LdVPhTOTgGyYbETktJPDuzXOij1bykMibNTgGwGllbM7PuS3tLnqbvc/btZ38jMDkg6IEnbtm3LPEAUY9C5JsOeeRIKOzuBwQYGubtfH+KN3P2IpCOSNDMzM6j1GAXLcq4J/dzAeKL9cIL1azd87uDevt/LzBgYXyPVyM3sQ2b2iqR3S3rCzI6FGRbyxmYboDpG7Vr5jrtf4e5/5O5vdvd9oQaGfHFRMVAddK1MKDbbANVBkE8oLioGqoMgr7C0m+zZbANUB10rFZV2dsrqDhRaCoH4EeQVNehiY4mWQqAqCPKIDHPJAouZwOSgRh6JYfu+WcwEJgdBHolh+75ZzAQmB6WVSAxbKmExE5gcBHkkNnLJAouZwGSgtBIJSiUAkjAjjwSlEgBJCPKIUCoB0A9BXpBhesABYBgEeQEGbZcHgFGw2FkAzv4GkCeCvABslweQJ4K8AGyXB5AngrwA9IADyBOLnQWgBxxAngjygtADDiAvlFYAIHIEOQBEjiAHgMgR5AAQORY7B+CMFADjjiBPwRkpAGJAaSUFZ6QAiAFBnoIzUgDEgCBPwRkpAGJAkKfgjBQAMWCxMwVnpACIAUE+AGekABh3lFYAIHIjBbmZHTazl8zsx2b2HTObCjUwAEA2o87In5L0dnd/h6SfSLpz9CEBAIYxUpC7+5PufmH5y+OSrhh9SACAYYSskX9a0veSnjSzA2Y2Z2Zz586dC/i2ADDZBnatmNn3Jb2lz1N3uft3l7/nLkkXJB1Neh13PyLpiCTNzMz4hkYLAFhnYJC7+/Vpz5vZpyR9QNJ73J2ABoCCjdRHbmY3SPq8pL909/NhhgQAGMaoNfL7Jb1J0lNmdtLMvhZgTACAIYw0I3f3Pws1kCy45AEA1otmiz6XPABAf9Fs0eeSBwDoL5og55IHAOgvmiDnkgcA6C+aIOeSBwDoL5rFTi55AID+oglyiUseAKCfaEorAID+CHIAiBxBDgCRI8gBIHIEOQBEjiAHgMhZGXdBmNk5ST8v/I2Ld5mkX5c9iALxeatrkj6rNL6f963uvrX3wVKCfFKY2Zy7z5Q9jqLweatrkj6rFN/npbQCAJEjyAEgcgR5vo6UPYCC8Xmra5I+qxTZ56VGDgCRY0YOAJEjyAEgcgR5zszssJm9ZGY/NrPvmNlU2WPKk5ndYmZnzOyimUXTvjUMM7vBzM6a2U/N7GDZ48mTmT1gZr8ysxfKHkvezOxKM3vGzP5r+f/hvy17TFkR5Pl7StLb3f0dkn4i6c6Sx5O3FyTdLOnZsgeSBzOrSfqqpPdJ2inpVjPbWe6ocvUNSTeUPYiCXJB0h7vvlHSdpM/G8t+WIM+Zuz/p7heWvzwu6Yoyx5M3d3/R3c+WPY4cvUvST939Z+7+B0nflvTBkseUG3d/VtJrZY+jCO7+S3f/0fK//17Si5KiuMmGIC/WpyV9r+xBYCTTkn6x6utXFMkfdmRnZtsl7Zb0g3JHkk1UV72NKzP7vqS39HnqLnf/7vL33KWlv7odLXJsecjyeYFYmdkbJT0q6TZ3/13Z48mCIA/A3a9Pe97MPiXpA5Le4xVo3B/0eSuuLenKVV9fsfwYKsDM6loK8aPu/ljZ48mK0krOzOwGSZ+XdJO7ny97PBjZDyW9zcyuMrNLJX1M0uMljwkBmJlJ+rqkF939S2WPZxgEef7ul/QmSU+Z2Ukz+1rZA8qTmX3IzF6R9G5JT5jZsbLHFNLywvXnJB3T0mLYQ+5+ptxR5cfMviXpeUk7zOwVM/tM2WPK0R5Jn5S0d/nP6kkze3/Zg8qCLfoAEDlm5AAQOYIcACJHkANA5AhyAIgcQQ4AkSPIASByBDkARO7/AQG+hN64fdMpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(x[:, 0], attributions[:, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "interactions = explainer.interactions(x,\n",
    "                                      multiply_by_input=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
